rm(list = ls())

### R package
# # only need to install once
# devtools::install_github("cloudyr/pyMTurkR", dependencies=TRUE)
# devtools::install_github("Luwei-Ying/validateIt", dependencies=TRUE)
# load R package
library(validateIt)

### load the crowdsourced results
# label intrusion (LI)
li_first <- vector() # first trial
li_second <- vector() # second trial

load("labelrecords/LInarrowcareful.Rdata")
result <- read.csv("labelresults/LInarrowcarefulresults1.csv", stringsAsFactors = FALSE)
li_first[1] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])
result <- read.csv("labelresults/LInarrowcarefulresults2.csv", stringsAsFactors = FALSE)
li_second[1] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])

load("labelrecords/LInarrowcursory.Rdata")
result <- read.csv("labelresults/LInarrowcursoryresults1.csv", stringsAsFactors = FALSE)
li_first[2] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])
result <- read.csv("labelresults/LInarrowcursoryresults2.csv", stringsAsFactors = FALSE)
li_second[2] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])

load("labelrecords/LIbroadcareful.Rdata")
result <- read.csv("labelresults/LIbroadcarefulresults1.csv", stringsAsFactors = FALSE)
li_first[3] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])
result <- read.csv("labelresults/LIbroadcarefulresults2.csv", stringsAsFactors = FALSE)
li_second[3] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])

load("labelrecords/LIbroadcursory.Rdata")
result <- read.csv("labelresults/LIbroadcursoryresults1.csv", stringsAsFactors = FALSE)
li_first[4] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])
result <- read.csv("labelresults/LIbroadcursoryresults2.csv", stringsAsFactors = FALSE)
li_second[4] <- as.numeric(evalResults(results = result, key = record, type = "LI")[[3]][1])

li_pool <- (li_first + li_second)/2

# results so far should be as follows:
# li_first <- c(0.634, 0.502, 0.85, 0.778)
# li_second <- c(0.618, 0.532, 0.818, 0.746)
# li_pool <- c(0.626, 0.517, 0.834, 0.762)

# optimal label (OL)
ol_first <- vector() # first trial
ol_second <- vector() # second trial

load("labelrecords/OLnarrowcareful.Rdata")
result <- read.csv("labelresults/OLnarrowcarefulresults1.csv", stringsAsFactors = FALSE)
ol_first[1] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])
result <- read.csv("labelresults/OLnarrowcarefulresults2.csv", stringsAsFactors = FALSE)
ol_second[1] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])

load("labelrecords/OLnarrowcursory.Rdata")
result <- read.csv("labelresults/OLnarrowcursoryresults1.csv", stringsAsFactors = FALSE)
ol_first[2] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])
result <- read.csv("labelresults/OLnarrowcursoryresults2.csv", stringsAsFactors = FALSE)
ol_second[2] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])

load("labelrecords/OLbroadcareful.Rdata")
result <- read.csv("labelresults/OLbroadcarefulresults1.csv", stringsAsFactors = FALSE)
ol_first[3] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])
result <- read.csv("labelresults/OLbroadcarefulresults2.csv", stringsAsFactors = FALSE)
ol_second[3] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])

load("labelrecords/OLbroadcursory.Rdata")
result <- read.csv("labelresults/OLbroadcursoryresults1.csv", stringsAsFactors = FALSE)
ol_first[4] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])
result <- read.csv("labelresults/OLbroadcursoryresults2.csv", stringsAsFactors = FALSE)
ol_second[4] <- as.numeric(evalResults(results = result, key = record, type = "OL")[[3]][1])

ol_pool <- (ol_first + ol_second)/2

# results so far should be as follows:
# ol_first <- c(0.788, 0.726, 0.858, 0.818)
# ol_second <- c(0.756, 0.702, 0.874, 0.814)
# ol_pool <- c(0.772, 0.714, 0.866, 0.816)

### generate Figure 4
# define transparent colors for the figure
library("ggplot2")
tomato_t <- alpha("tomato", 0.35)
chartreuse4_t <- alpha("chartreuse4", 0.35)
tomato <- "tomato"
chartreuse4 <- "chartreuse4"

# output plot in pdf
pdf("labelplots/figure4.pdf", width = 10, height = 7)
par(mgp = c(1.5, 0, 0), mar = c(2, 3, .7, .7))
plot(NULL,
     main = NA,
     ylim=c(0, 1.02), 
     xlim = c(0.6, 2.4), 
     ylab = "Proportion Correct",
     xlab = NA,
     cex.lab = 1.2,
     axes = F)
axis(side = 2, at = seq(0, 1, by = 0.2), col.ticks = NA, cex.axis = 1.2)
axis(side = 1, at = 1,
     labels = c('Label Intrusion'), 
     las = 1,
     col = NA,
     col.ticks = NA,
     cex.axis = 1.2)
text(x = 0.77, y = 0.025, cex = 1.2,
     labels = "Within Category")
text(x = 1.15, y = 0.025, cex = 1.2,
     labels = "Across Categories")
axis(side = 1, at = 2,
     labels = c('Optimal Label'),
     las = 1,
     col = NA,
     col.ticks = NA,
     cex.axis = 1.2)
text(x = 1.77, y = 0.025, cex = 1.2,
     labels = "Within Category")
text(x = 2.15, y = 0.025, cex = 1.2,
     labels = "Across Categories")
legend(0.55, 1.03,
       c("Careful Coder",
         "Cursory Coder"),
       col = c(chartreuse4, tomato),
       lty = 1,
       lwd = 3,
       cex = 1.2,
       bty = 'n')
# ----------------------------------------------------------------
abline(h = 0.25, col = "gray", lty = 1, lwd = 2)
# -------------------------------- LI ---------------------------------
# narrowcareful
points(x = 0.7, y = li_first[1], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 0.7, y0 = li_first[1]-1.96*sqrt(li_first[1]*(1-li_first[1])/500), x1 = 0.7, y1 = li_first[1]+1.96*sqrt(li_first[1]*(1-li_first[1])/500), col = chartreuse4_t, lwd = 3)
points(x = 0.72, y = li_second[1], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 0.72, y0 = li_second[1]-1.96*sqrt(li_second[1]*(1-li_second[1])/500), x1 = 0.72, y1 = li_second[1]+1.96*sqrt(li_second[1]*(1-li_second[1])/500), col = chartreuse4_t, lwd = 3)
points(x = 0.75, y = li_pool[1], pch = 20, col = chartreuse4, cex = 2)
segments(x0 = 0.75, y0 = li_pool[1]-1.96*sqrt(li_pool[1]*(1-li_pool[1])/1000), x1 = 0.75, y1 = li_pool[1]+1.96*sqrt(li_pool[1]*(1-li_pool[1])/1000), col = chartreuse4, lwd = 3)
# narrowcursory
points(x = 0.85, y = li_first[2], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 0.85, y0 = li_first[2]-1.96*sqrt(li_first[2]*(1-li_first[2])/500), x1 = 0.85, y1 = li_first[2]+1.96*sqrt(li_first[2]*(1-li_first[2])/500), col = tomato_t, lwd = 3)
points(x = 0.87, y = li_second[2], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 0.87, y0 = li_second[2]-1.96*sqrt(li_second[2]*(1-li_second[2])/500), x1 = 0.87, y1 = li_second[2]+1.96*sqrt(li_second[2]*(1-li_second[2])/500), col = tomato_t, lwd = 3)
points(x = 0.9, y = li_pool[2], pch = 20, col = tomato, cex = 2)
segments(x0 = 0.9, y0 = li_pool[2]-1.96*sqrt(li_pool[2]*(1-li_pool[2])/1000), x1 = 0.9, y1 = li_pool[2]+1.96*sqrt(li_pool[2]*(1-li_pool[2])/1000), col = tomato, lwd = 3)
# ----------------------------------------------------------------
# broadcareful
points(x = 1, y = li_first[3], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 1, y0 = li_first[3]-1.96*sqrt(li_first[3]*(1-li_first[3])/500), x1 = 1, y1 = li_first[3]+1.96*sqrt(li_first[3]*(1-li_first[3])/500), col = chartreuse4_t, lwd = 3)
points(x = 1.02, y = li_second[3], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 1.02, y0 = li_second[3]-1.96*sqrt(li_second[3]*(1-li_second[3])/500), x1 = 1.02, y1 = li_second[3]+1.96*sqrt(li_second[3]*(1-li_second[3])/500), col = chartreuse4_t, lwd = 3)
points(x = 1.05, y = li_pool[3], pch = 20, col = chartreuse4, cex = 2)
segments(x0 = 1.05, y0 = li_pool[3]-1.96*sqrt(li_pool[3]*(1-li_pool[3])/1000), x1 = 1.05, y1 = li_pool[3]+1.96*sqrt(li_pool[3]*(1-li_pool[3])/1000), col = chartreuse4, lwd = 3)
# broadcursory
points(x = 1.15, y = li_first[4], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 1.15, y0 = li_first[4]-1.96*sqrt(li_first[4]*(1-li_first[4])/500), x1 = 1.15, y1 = li_first[4]+1.96*sqrt(li_first[4]*(1-li_first[4])/500), col = tomato_t, lwd = 3)
points(x = 1.17, y = li_second[4], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 1.17, y0 = li_second[4]-1.96*sqrt(li_second[4]*(1-li_second[4])/500), x1 = 1.17, y1 = li_second[4]+1.96*sqrt(li_second[4]*(1-li_second[4])/500), col = tomato_t, lwd = 3)
points(x = 1.2, y = li_pool[4], pch = 20, col = tomato, cex = 2)
segments(x0 = 1.2, y0 = li_pool[4]-1.96*sqrt(li_pool[4]*(1-li_pool[4])/1000), x1 = 1.2, y1 = li_pool[4]+1.96*sqrt(li_pool[4]*(1-li_pool[4])/1000), col = tomato, lwd = 3)
# ------------------------------- p-values -------------------------------
segments(x0 = 0.75, y0 = 0.4, x1 = 0.75, y1 = 0.58, col = "gray", lty = 3)
segments(x0 = 0.75, y0 = 0.4, x1 = 0.9, y1 = 0.4, col = "gray", lty = 3)
segments(x0 = 0.9, y0 = 0.4, x1 = 0.9, y1 = 0.47, col = "gray", lty = 3)
text(x = 0.83, y = 0.37, "0.000")
segments(x0 = 1.05, y0 = 0.65, x1 = 1.05, y1 = 0.79, col = "gray", lty = 3)
segments(x0 = 1.05, y0 = 0.65, x1 = 1.2, y1 = 0.65, col = "gray", lty = 3)
segments(x0 = 1.2, y0 = 0.65, x1 = 1.2, y1 = 0.715, col = "gray", lty = 3)
text(x = 1.13, y = 0.62, "0.000")
# -------------------------------- OL ---------------------------------
# narrowcareful
points(x = 1.7, y = ol_first[1], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 1.7, y0 = ol_first[1]-1.96*sqrt(ol_first[1]*(1-ol_first[1])/456), x1 = 1.7, y1 = ol_first[1]+1.96*sqrt(ol_first[1]*(1-ol_first[1])/456), col = chartreuse4_t, lwd = 3)
points(x = 1.72, y = ol_second[1], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 1.72, y0 = ol_second[1]-1.96*sqrt(ol_second[1]*(1-ol_second[1])/500), x1 = 1.72, y1 = ol_second[1]+1.96*sqrt(ol_second[1]*(1-ol_second[1])/500), col = chartreuse4_t, lwd = 3)
points(x = 1.75, y = ol_pool[1], pch = 20, col = chartreuse4, cex = 2)
segments(x0 = 1.75, y0 = ol_pool[1]-1.96*sqrt(ol_pool[1]*(1-ol_pool[1])/1000), x1 = 1.75, y1 = ol_pool[1]+1.96*sqrt(ol_pool[1]*(1-ol_pool[1])/1000), col = chartreuse4, lwd = 3)
# narrowcursory
points(x = 1.85, y = ol_first[2], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 1.85, y0 = ol_first[2]-1.96*sqrt(ol_first[2]*(1-ol_first[2])/500), x1 = 1.85, y1 = ol_first[2]+1.96*sqrt(ol_first[2]*(1-ol_first[2])/500), col = tomato_t, lwd = 3)
points(x = 1.87, y = ol_second[2], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 1.87, y0 = ol_second[2]-1.96*sqrt(ol_second[2]*(1-ol_second[2])/500), x1 = 1.87, y1 = ol_second[2]+1.96*sqrt(ol_second[2]*(1-ol_second[2])/500), col = tomato_t, lwd = 3)
points(x = 1.9, y = ol_pool[2], pch = 20, col = tomato, cex = 2)
segments(x0 = 1.9, y0 = ol_pool[2]-1.96*sqrt(ol_pool[2]*(1-ol_pool[2])/1000), x1 = 1.9, y1 = ol_pool[2]+1.96*sqrt(ol_pool[2]*(1-ol_pool[2])/1000), col = tomato, lwd = 3)
# ----------------------------------------------------------------
# broadcareful
points(x = 2, y = ol_first[3], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 2, y0 = ol_first[3]-1.96*sqrt(ol_first[3]*(1-ol_first[3])/500), x1 = 2, y1 = ol_first[3]+1.96*sqrt(ol_first[3]*(1-ol_first[3])/500), col = chartreuse4_t, lwd = 3)
points(x = 2.02, y = ol_second[3], pch = 20, col = chartreuse4_t, cex = 2)
segments(x0 = 2.02, y0 = ol_second[3]-1.96*sqrt(ol_second[3]*(1-ol_second[3])/500), x1 = 2.02, y1 = ol_second[3]+1.96*sqrt(ol_second[3]*(1-ol_second[3])/500), col = chartreuse4_t, lwd = 3)
points(x = 2.05, y = ol_pool[3], pch = 20, col = chartreuse4, cex = 2)
segments(x0 = 2.05, y0 = ol_pool[3]-1.96*sqrt(ol_pool[3]*(1-ol_pool[3])/1000), x1 = 2.05, y1 = ol_pool[3]+1.96*sqrt(ol_pool[3]*(1-ol_pool[3])/1000), col = chartreuse4, lwd = 3)
# broadcursory
points(x = 2.15, y = ol_first[4], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 2.15, y0 = ol_first[4]-1.96*sqrt(ol_first[4]*(1-ol_first[4])/500), x1 = 2.15, y1 = ol_first[4]+1.96*sqrt(ol_first[4]*(1-ol_first[4])/500), col = tomato_t, lwd = 3)
points(x = 2.17, y = ol_second[4], pch = 20, col = tomato_t, cex = 2)
segments(x0 = 2.17, y0 = ol_second[4]-1.96*sqrt(ol_second[4]*(1-ol_second[4])/500), x1 = 2.17, y1 = ol_second[4]+1.96*sqrt(ol_second[4]*(1-ol_second[4])/500), col = tomato_t, lwd = 3)
points(x = 2.2, y = ol_pool[4], pch = 20, col = tomato, cex = 2)
segments(x0 = 2.2, y0 = ol_pool[4]-1.96*sqrt(ol_pool[4]*(1-ol_pool[4])/1000), x1 = 2.2, y1 = ol_pool[4]+1.96*sqrt(ol_pool[4]*(1-ol_pool[4])/1000), col = tomato, lwd = 3)
# ------------------------------- p-values -------------------------------
segments(x0 = 1.75, y0 = 0.62, x1 = 1.75, y1 = 0.74, col = "gray", lty = 3)
segments(x0 = 1.75, y0 = 0.62, x1 = 1.9, y1 = 0.62, col = "gray", lty = 3)
segments(x0 = 1.9, y0 = 0.62, x1 = 1.9, y1 = 0.67, col = "gray", lty = 3)
text(x = 1.83, y = 0.59, "0.004")
segments(x0 = 2.05, y0 = 0.7, x1 = 2.05, y1 = 0.83, col = "gray", lty = 3)
segments(x0 = 2.05, y0 = 0.7, x1 = 2.2, y1 = 0.7, col = "gray", lty = 3)
segments(x0 = 2.2, y0 = 0.7, x1 = 2.2, y1 = 0.78, col = "gray", lty = 3)
text(x = 2.13, y = 0.67, "0.003")
dev.off()

### generate table 4 (Accuracy within and across Two Broader Categories)
# define a function
getIndicator <- function(key, results){
        indicator <- NULL
        for(i in 1:nrow(results)){
                correct <- as.vector(key[[2]][i,-1])[as.numeric(results[i,6])] == as.character(key[[1]][i, (ncol(key[[1]])-1)])
                indicator <- c(indicator, correct)
        }
        return(indicator)
}
# across-categorical labels
Internationalcareful <- c("International Trade",
                          "Praising Active Military/Military Units",
                          "Terrorism",
                          "Military Sexual Assault",
                          "Nuclear Deterrance/International Security",
                          "Air Force",
                          "Honoring Specific Veterans",
                          "Honoring Veterans/Heroes",
                          "Military Operations/Armed Conflicts",
                          "Veterans Affairs/Veterans Healthcare")
Internationalcursory <- c("Manufacturing",
                          "\"Welcome Home\" Messages",
                          "Islamic Extremists",
                          "Military Affairs",
                          "Foreign Affairs",
                          "Military",
                          "Military Service",
                          "\"Thank you\" Messages",
                          "Counter-terrorism",
                          "Veterans")
table4 <- matrix(NA, nrow = 4, ncol = 4)
# LI
load("labelrecords/LIbroadcareful.Rdata")
LIbroadcarefulresults1 <- read.csv("labelresults/LIbroadcarefulresults1.csv", stringsAsFactors = FALSE)
LIbroadcarefulresults2 <- read.csv("labelresults/LIbroadcarefulresults2.csv", stringsAsFactors = FALSE)

LIbroadcarefulresults1$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
LIbroadcarefulresults1$cross <- ifelse(record[[1]][,6] %in% Internationalcareful, 1, 0)
LIbroadcarefulresults1$indicator <- getIndicator(record, LIbroadcarefulresults1)
LIbroadcarefulresults1 <- LIbroadcarefulresults1[LIbroadcarefulresults1$gold == 0,]
LIbroadcarefulresults2$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
LIbroadcarefulresults2$cross <- ifelse(record[[1]][,6] %in% Internationalcareful, 1, 0)
LIbroadcarefulresults2$indicator <- getIndicator(record, LIbroadcarefulresults2)
LIbroadcarefulresults2 <- LIbroadcarefulresults2[LIbroadcarefulresults2$gold == 0,]

tab <- table(c(LIbroadcarefulresults1$indicator, LIbroadcarefulresults2$indicator), 
             c(LIbroadcarefulresults1$cross, LIbroadcarefulresults2$cross))
table4[1,1:2] <- tab[2,]/colSums(tab)

load("labelrecords/LIbroadcursory.Rdata")
LIbroadcursoryresults1 <- read.csv("labelresults/LIbroadcursoryresults1.csv", stringsAsFactors = FALSE)
LIbroadcursoryresults2 <- read.csv("labelresults/LIbroadcursoryresults2.csv", stringsAsFactors = FALSE)

LIbroadcursoryresults1$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
LIbroadcursoryresults1$cross <- ifelse(record[[1]][,6] %in% Internationalcursory, 1, 0)
LIbroadcursoryresults1$indicator <- getIndicator(record, LIbroadcursoryresults1)
LIbroadcursoryresults1 <- LIbroadcursoryresults1[LIbroadcursoryresults1$gold == 0,]
LIbroadcursoryresults2$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
LIbroadcursoryresults2$cross <- ifelse(record[[1]][,6] %in% Internationalcursory, 1, 0)
LIbroadcursoryresults2$indicator <- getIndicator(record, LIbroadcursoryresults2)
LIbroadcursoryresults2 <- LIbroadcursoryresults2[LIbroadcursoryresults2$gold == 0,]

tab <- table(c(LIbroadcursoryresults1$indicator, LIbroadcursoryresults2$indicator), 
             c(LIbroadcursoryresults1$cross, LIbroadcursoryresults2$cross))
table4[2,1:2] <- tab[2,]/colSums(tab)

# OL
load("labelrecords/OLbroadcareful.Rdata")
OLbroadcarefulresults1 <- read.csv("labelresults/OLbroadcarefulresults1.csv", stringsAsFactors = FALSE)
OLbroadcarefulresults2 <- read.csv("labelresults/OLbroadcarefulresults2.csv", stringsAsFactors = FALSE)

OLbroadcarefulresults1$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
for (i in 1:nrow(record[[1]])){
        OLbroadcarefulresults1$cross[i] <- sum(record[[1]][i,3] %in% Internationalcareful,
                                           record[[1]][i,4] %in% Internationalcareful,
                                           record[[1]][i,5] %in% Internationalcareful)
}
OLbroadcarefulresults1$indicator <- getIndicator(record, OLbroadcarefulresults1)
OLbroadcarefulresults1 <- OLbroadcarefulresults1[OLbroadcarefulresults1$gold == 0,]

OLbroadcarefulresults2$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
for (i in 1:nrow(record[[1]])){
        OLbroadcarefulresults2$cross[i] <- sum(record[[1]][i,3] %in% Internationalcareful,
                                           record[[1]][i,4] %in% Internationalcareful,
                                           record[[1]][i,5] %in% Internationalcareful)
}
OLbroadcarefulresults2$indicator <- getIndicator(record, OLbroadcarefulresults2)
OLbroadcarefulresults2 <- OLbroadcarefulresults2[OLbroadcarefulresults2$gold == 0,]

tab <- table(c(OLbroadcarefulresults1$indicator, OLbroadcarefulresults2$indicator), 
             c(OLbroadcarefulresults1$cross, OLbroadcarefulresults2$cross))
table4[3,] <- tab[2,]/colSums(tab)

load("labelrecords/OLbroadcursory.Rdata")
OLbroadcursoryresults1 <- read.csv("labelresults/OLbroadcursoryresults1.csv", stringsAsFactors = FALSE)
OLbroadcursoryresults2 <- read.csv("labelresults/OLbroadcursoryresults2.csv", stringsAsFactors = FALSE)

OLbroadcursoryresults1$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
for (i in 1:nrow(record[[1]])){
        OLbroadcursoryresults1$cross[i] <- sum(record[[1]][i,3] %in% Internationalcursory,
                                               record[[1]][i,4] %in% Internationalcursory,
                                               record[[1]][i,5] %in% Internationalcursory)
}
OLbroadcursoryresults1$indicator <- getIndicator(record, OLbroadcursoryresults1)
OLbroadcursoryresults1 <- OLbroadcursoryresults1[OLbroadcursoryresults1$gold == 0,]


OLbroadcursoryresults2$gold <- ifelse(record[[1]][,1] == "gold", 1, 0)
for (i in 1:nrow(record[[1]])){
        OLbroadcursoryresults2$cross[i] <- sum(record[[1]][i,3] %in% Internationalcursory,
                                               record[[1]][i,4] %in% Internationalcursory,
                                               record[[1]][i,5] %in% Internationalcursory)
}
OLbroadcursoryresults2$indicator <- getIndicator(record, OLbroadcursoryresults2)
OLbroadcursoryresults2 <- OLbroadcursoryresults2[OLbroadcursoryresults2$gold == 0,]

tab <- table(c(OLbroadcursoryresults1$indicator, OLbroadcursoryresults2$indicator), 
             c(OLbroadcursoryresults1$cross, OLbroadcursoryresults2$cross))
table4[4,] <- tab[2,]/colSums(tab)

row.names(table4) <- c("LIcareful", "LIcursory", "OLcareful", "OLcursory")

# output results for table 4
write.table(table4, "labeltables/table4.txt")
